from __future__ import division

# This program works OK tonight. 2012/11/27

# questions?
# what is the final(so far) training file produced?

# The files dealing with this time are:
# 
inputFileName = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2/wholeTrainingFileTempleteGov2V1_phase0_posting_oriented_term_freq_inCollection_added.txt.input"
inputFileHandler = open(inputFileName,"r")

inputFileName2 = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/rawTermInvertedIndexOutput.txt"
inputFileHandler2 = open(inputFileName2,"r")

inputAuxFileName = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/rawTermInvertedIndexOutputAccessAuxFile_NEW.txt"
inputAuxFileHandler = open(inputAuxFileName,"r")

outputErrorFileName = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/errorMessageForAddingRankInfoToTrainingFile.txt"
outputErrorFileHandler = open(outputErrorFileName,"w")

outputFileName = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2/wholeTrainingFileTempleteGov2V1_phase0_posting_oriented_posting_rank_in_its_index_added.txt.input"
outputFileHandler = open(outputFileName,"w")

queryTermAccessIndexAuxDict = {}

for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if len(lineElements) == 5:
        queryTermIndex = lineElements[0]
        queryTerm = lineElements[1]
        freqOfQueryTermInCollection = lineElements[2]
        invertedIndexBeginningPosition = lineElements[3]
        invertedIndexEndingPosition = lineElements[4]
        
        if queryTerm not in queryTermAccessIndexAuxDict:
            queryTermAccessIndexAuxDict[queryTerm] = freqOfQueryTermInCollection + " " + invertedIndexBeginningPosition + " " + invertedIndexEndingPosition
        
    else:
        # for the situation when the file is still modifying
        pass

print "len(queryTermAccessIndexAuxDict):",len(queryTermAccessIndexAuxDict)
outputErrorFileHandler.write("The following query terms can NOT find there index entry in the index." + "\n")

termNotFoundList = []


for line in inputFileHandler.readlines():
    # Example line dealt with: 
    # 701 GX007-14-8938106 history 2.010177 2279369.000000 6.000000 1842 11.304873 17 464 NOTTOP10
    # OK, it is because of me
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[2]
    if queryTerm not in queryTermAccessIndexAuxDict:
        print "queryTerm:",queryTerm
        if queryTerm not in termNotFoundList:
            termNotFoundList.append(queryTerm)
    else:
        # Here, let's do the counting of the following problem. What is the rank for this posting in the inverted index
        elements = queryTermAccessIndexAuxDict[queryTerm].strip().split(" ")
        freqOfQueryTermInCollection = elements[0]
        invertedIndexBeginningPosition = elements[1]
        invertedIndexEndingPosition = elements[2]
        
        # Some serious data inconsistency problems happened here!! And so force me to think of another way.
        # Updated 2012/11/27 night
    
    

termNotFoundList.sort(cmp=None, key=None, reverse=False)
for term in termNotFoundList:
    outputErrorFileHandler.write(term + "\n")

outputErrorFileHandler.close()
inputFileHandler.close()
outputFileHandler.close()

    
    
